# -*- coding: utf-8 -*-
import scrapy
import scrapy
import sys
sys.path.append('..')
from movie.items import BlogItem

class BlogSpider(scrapy.Spider):
    name = 'blog'
    allowed_domains = ['www.cnblogs.com']
    start_urls = ['https://www.cnblogs.com/kevingrace/']

    def parse(self, response):
        blogItems = response.xpath('//div[@class="forFlow"]/div[@class="day"]/div[@class="postTitle"]')
        for subBlog in blogItems:
            title = subBlog.xpath('./a[@class="postTitle2"]/text()').extract()[0]
            url = subBlog.xpath('./a[@class="postTitle2"]/@href').extract()[0]
            date = subBlog.xpath('../div[@class="dayTitle"]/a/text()').extract()[0]
            item = BlogItem()
            item['title'] = title
            item['url'] = url
            item['date'] = date
            yield scrapy.Request(url=item['url'], meta={"item": item}, callback=self.parseContent)
        next_url = response.xpath('//a[contains(text(), "下一页")]/@href')
        if next_url:
            next_url = next_url.extract()[0]
            yield scrapy.Request(url=next_url, callback=self.parse)

    def parseContent(self, response):
        item = response.meta['item']
        item['content'] = ''.join(response.xpath('//div[@id="cnblogs_post_body"]/node()').extract())
        yield item